/*
 * Copyright (C) 2014 hu
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */
package com.sentiment.webcollector.crawler;

import java.util.ArrayList;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.sentiment.config.ConfigDb;
import com.sentiment.database.bean.Datum;
import com.sentiment.database.codec.DatumCodec;
import com.sentiment.database.dbUtil.DbCodecDriver;
import com.sentiment.webcollector.fetcher.DbUpdater;
import com.sentiment.webcollector.fetcher.Fetcher;
import com.sentiment.webcollector.fetcher.VisitorFactory;
import com.sentiment.webcollector.generator.Injector;
import com.sentiment.webcollector.generator.StandardGenerator;
import com.sentiment.webcollector.net.HttpRequester;
import com.sentiment.webcollector.net.HttpRequesterImpl;
import com.sentiment.webcollector.util.Config;

/**
 *
 * @author hu
 */
public abstract class Crawler implements VisitorFactory {

	public static final Logger LOG = LoggerFactory.getLogger(Crawler.class);

	protected int status;
	public final static int RUNNING = 1;
	public final static int STOPED = 2;
	protected boolean resumable = false;
	protected int threads = 50;
	protected Integer topN = null;
	protected ArrayList<String> seeds = new ArrayList<String>();
	protected ArrayList<String> forcedSeeds = new ArrayList<String>();
	protected Fetcher fetcher;
	int retry = Config.retry;
	protected int maxRetry = Config.MAX_RETRY;

	protected VisitorFactory visitorFactory = this;
	protected HttpRequester httpRequester = new HttpRequesterImpl();
	protected DbCodecDriver<Datum> driver = new DbCodecDriver<Datum>();
	public Crawler() {
		
	}

	public void inject() throws Exception {
		Injector injector = new Injector();
		injector.inject(seeds);
	}

	public void injectForcedSeeds() throws Exception {
		Injector injector = new Injector();
		injector.inject(forcedSeeds);
	}

	public void start(int depth) throws Exception {
		driver.openConnection(ConfigDb.crawlerDb, Datum.class, DatumCodec.class);
		boolean needInject = true;

		if (resumable) {
			needInject = false;
		}
		if (!resumable) {
			if (seeds.isEmpty() && forcedSeeds.isEmpty()) {
				LOG.info("error:Please add at least one seed");
				return;
			}
		}
		if (needInject) {
			inject();
		}

		if (!forcedSeeds.isEmpty()) {
			injectForcedSeeds();
		}

		status = RUNNING;
		for (int i = 0; i < depth; i++) {
			if (status == STOPED) {
				break;
			}
			LOG.info("starting depth " + (i + 1));
			long startTime = System.currentTimeMillis();

			StandardGenerator generator = new StandardGenerator(driver);
			generator.setMaxRetry(maxRetry);
			generator.setTopN(topN);
			fetcher = new Fetcher();
			fetcher.setHttpRequester(httpRequester);
			fetcher.setDbUpdater(new DbUpdater(driver));
			fetcher.setVisitorFactory(visitorFactory);
			fetcher.setRetry(retry);
			fetcher.setThreads(threads);
			fetcher.fetchAll(generator);
			long endTime = System.currentTimeMillis();
			long costTime = (endTime - startTime) / 1000;
			int totalGenerate = generator.getTotalGenerate();

			LOG.info("depth " + (i + 1) + " finish: \n\tTOTAL urls:\t" + totalGenerate + "\n\tTOTAL time:\t" + costTime
					+ " seconds");
			if (totalGenerate == 0) {
				break;
			}

		}
	}

	public void stop() {
		status = STOPED;
		fetcher.stop();
	}

	public VisitorFactory getVisitorFactory() {
		return visitorFactory;
	}

	public void setVisitorFactory(VisitorFactory visitorFactory) {
		this.visitorFactory = visitorFactory;
	}

	public HttpRequester getHttpRequester() {
		return httpRequester;
	}

	public void setHttpRequester(HttpRequester httpRequester) {
		this.httpRequester = httpRequester;
	}

	/**
	 * 添加一个种子url(如果断点爬取，种子只会在第一次爬取时注入)
	 *
	 * @param seed
	 *            种子url
	 */
	public void addSeed(String seed) {
		seeds.add(seed);
	}

	/**
	 * 添加一个种子url(如果断点爬取，种子会在每次启动爬虫时注入， 如果爬取历史中有相同url,则覆盖)
	 *
	 * @param seed
	 */
	public void addForcedSeed(String seed) {
		forcedSeeds.add(seed);
	}

	public ArrayList<String> getSeeds() {
		return seeds;
	}

	public void setSeeds(ArrayList<String> seeds) {
		this.seeds = seeds;
	}

	public ArrayList<String> getForcedSeeds() {
		return forcedSeeds;
	}

	public void setForcedSeeds(ArrayList<String> forcedSeeds) {
		this.forcedSeeds = forcedSeeds;
	}

	public boolean isResumable() {
		return resumable;
	}

	public void setResumable(boolean resumable) {
		this.resumable = resumable;
	}

	public int getThreads() {
		return threads;
	}

	public void setThreads(int threads) {
		this.threads = threads;
	}

	public Integer getTopN() {
		return topN;
	}

	public void setTopN(Integer topN) {
		this.topN = topN;
	}

	public int getRetry() {
		return retry;
	}

	public void setRetry(int retry) {
		this.retry = retry;
	}

	public int getMaxRetry() {
		return maxRetry;
	}

	public void setMaxRetry(int maxRetry) {
		this.maxRetry = maxRetry;
	}

}
